# -*- coding:utf-8 -*-

import requests
from lxml import etree
from fake_useragent import UserAgent
from http import cookiejar
import json
from datetime import datetime
import time
import re
import random
from pymongo import MongoClient


requests.packages.urllib3.disable_warnings()  # 忽略HTTPS安全警告

"""
链家-租房
https://bj.lianjia.com/zufang/
"""


class Test_Get():
    def __init__(self):
        #声明一个CookieJar对象实例来保存cookie
        self.cookie = cookiejar.CookieJar()
        ua = UserAgent(use_cache_server=False)  # 禁用服务器缓存
        self.headers = {
            # "User-Agent": ua.random
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
            "Accept-Encoding": "gzip, deflate, br",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Connection": "keep-alive",
            "Cookie": "lianjia_uuid=7143f19b-db1d-40dc-a8f2-608768234249; UM_distinctid=170487adbf618-042b00c455fafd-313f69-144000-170487adbf73b; _jzqc=1; _jzqx=1.1581763976.1581763976.1.jzqsr=bj%2Elianjia%2Ecom|jzqct=/zufang/pg2/.-; _jzqckmp=1; sajssdk_2015_cross_new_user=1; _smt_uid=5e47cd88.4ff6acd3; _ga=GA1.2.929592675.1581763980; _gid=GA1.2.1616947165.1581763980; _qzjc=1; digv_extends=%7B%22utmTrackId%22%3A%2221583074%22%7D; lianjia_ssid=e65bf620-8ef4-4d7b-bc57-3aab18de2014; _jzqa=1.2933291601844614700.1581763976.1581763976.1581778235.2; _jzqy=1.1581778235.1581778235.1.jzqsr=baidu|jzqct=%E9%93%BE%E5%AE%B6.-; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22170487adcdd805-0c8a7b09aed53e-313f69-1327104-170487adcde5ec%22%2C%22%24device_id%22%3A%22170487adcdd805-0c8a7b09aed53e-313f69-1327104-170487adcde5ec%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E4%BB%98%E8%B4%B9%E5%B9%BF%E5%91%8A%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22https%3A%2F%2Fsp0.baidu.com%2F9q9JcDHa2gU2pMbgoY3K%2Fadrc.php%3Ft%3D06KL00c00fZg9KY0DvNb0nVfAs0vKR9X000005tSQ7C00000ILwQAf.THLKVQ1i0A3qmh7GuZR0T1d9nj9BPW04uH0snj0vPj9b0ZRqPjTdwWbznW0YwH9Dn1-7nbD3fW7DrHu7nHDvwj7arDf%22%2C%22%24latest_referrer_host%22%3A%22sp0.baidu.com%22%2C%22%24latest_search_keyword%22%3A%22%E9%93%BE%E5%AE%B6%22%2C%22%24latest_utm_source%22%3A%22baidu%22%2C%22%24latest_utm_medium%22%3A%22pinzhuan%22%2C%22%24latest_utm_campaign%22%3A%22sousuo%22%2C%22%24latest_utm_content%22%3A%22biaotimiaoshu%22%2C%22%24latest_utm_term%22%3A%22biaoti%22%7D%7D; select_city=440300; _qzja=1.1883095760.1581763998730.1581763998730.1581778246832.1581763998730.1581778246832.0.0.0.2.2; _qzjto=2.2.0; _jzqb=1.3.10.1581778235.1; CNZZDATA1255849469=1175143247-1581761701-https%253A%252F%252Fwww.lianjia.com%252F%7C1581777914; CNZZDATA1254525948=678736726-1581760536-https%253A%252F%252Fwww.lianjia.com%252F%7C1581776736; CNZZDATA1255633284=149259611-1581763949-https%253A%252F%252Fwww.lianjia.com%252F%7C1581773747; CNZZDATA1255604082=562148569-1581760816-https%253A%252F%252Fwww.lianjia.com%252F%7C1581776699; _qzjb=1.1581778246831.1.0.0.0; srcid=eyJ0Ijoie1wiZGF0YVwiOlwiY2QzNDIyYjg1ODQ1MWJhYjBiM2U0ZjEwYmY1NzI0YTU5MTAyMWM0NWUyMTdjZGE4ZjY3MmY5YmNkZWIxNmJkYzIwMTgzMTA1ZjQyZWU0YTdmMWU1NDc3ZjEyMzhjNDQwYjUxNzdhNDU5OWNkMzk4OTkxMmU4MmExNTg4MzM2OGQ1N2VlNzZkNGIwN2MwNDVlMDEzNzQ3M2NmMmUwODkwMTFmYTlkNThjY2MzZjk4MmJkOTNkODdmMWFmNjEzZWY4NWZhNTAwNTQ5MTQ5ZWNiZTIxMjhhZTgzMjAyYzMwZDdmZTZlYjNjZTRhOTMwMTNhMjFiYzNlZGJjYmE4ODM0OWE0ZjMwNTc2N2I3YTYzYzZhYzFmYWVkOWI4MjQ2MTRjNTg0NmMwMjE4NmFhNDI0ZjQ1YTA5NDg3MDYzMTlkZjMyYWQ2OGFmOTg1Yzc1NDZhYzhmYzg0MTU2NTI3ZTM4Y1wiLFwia2V5X2lkXCI6XCIxXCIsXCJzaWduXCI6XCIzZTU3MGYxMlwifSIsInIiOiJodHRwczovL3N6LmxpYW5qaWEuY29tL3p1ZmFuZy9sdW9odXF1L2FiMjAwMzAxMDAxMDAwcGcxIiwib3MiOiJ3ZWIiLCJ2IjoiMC4xIn0=",
            "DNT": "1",
            "Host": "sz.lianjia.com",
            "Sec-Fetch-Mode": "navigate",
            "Sec-Fetch-Site": "same-origin",
            "Sec-Fetch-User": "?1",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"
        }

    def get_contents(self, current, url, area):
        postData = {

        }
        # url = f"https://bj.lianjia.com/zufang/pg{current}/"
        html = HttpUtils.do_request("GET", url, self.headers, "")
        root = etree.HTML(html.text)
        for s in root.xpath('//div[@class="content__list--item"]'):
            dict_data = dict()
            # 区域
            dict_data['quyu'] = area
            dict_data['url'] = "https://sz.lianjia.com" + s.xpath('./div/p[1]/a/@href')[0]
            dict_data['pic_url'] = s.xpath('./a/img/@data-src')[0]
            dict_data['title'] = s.xpath('./div/p[1]/a/text()')[0].replace("\n", "").strip()  # 标题
            tmp = "".join(s.xpath('./div/p[2]//text()')).replace("\n", "").split("/")
            dict_data['address'] = tmp[0].strip()  # 地址
            dict_data['lx'] = tmp[1].strip() + tmp[2].strip() + tmp[3].strip()  # 房屋类型
            dict_data['price'] = "".join(s.xpath('./div/span//text()'))  # 价格
            dict_data['whsj'] = "".join(s.xpath('./div/p[@class="content__list--item--brand oneline"]/span[2]//text()'))
            # 标签
            num = 0
            for d in s.xpath('./div[1]/p[@class="content__list--item--bottom oneline"]/i/text()'):
                dict_data['tag_' + str(num)] = d
                num += 1
            # print(str(dict_data))
            self.detail(dict_data['url'], dict_data)

    # 详细信息
    def detail(self, url, dict_data):
        html = HttpUtils.do_request("GET", url, self.headers, "")
        time.sleep(random.randint(1, 1))
        root = etree.HTML(html.text)
        # dict_data = dict()
        # 经纬度
        location_tmp = re.findall(r"g_conf.coord = \{(.*?)\}", html.text, re.M|re.S)[0]\
            .replace("\n", "").replace('\'', '').strip()
        location_tmp = location_tmp.replace("longitude:", "").replace("latitude:", "").replace(" ", "")
        dict_data['location'] = location_tmp
        # 地铁
        ditie_tmp = []
        for di in root.xpath('//div[@id="around"]/ul/li'):
            ditie_tmp.append(" ".join(di.xpath('./span//text()')))
        dict_data['ditie'] = "|".join(ditie_tmp)
        # 房源维护时间
        dict_data['whsj'] = root.xpath('//div[@class="content__subtitle"]/text()')[0].replace("\n", "").strip()
        # 房源编号
        dict_data['bh'] = "".join(root.xpath('//div[@class="content__subtitle"]/i/text()')).strip()
        # 租赁方式
        dict_data['fs'] = "".join(root.xpath('//ul[@class="content__aside__list"]/li[1]//text()'))
        # 房屋类型
        dict_data['lx'] = "".join(root.xpath('//ul[@class="content__aside__list"]/li[2]//text()'))
        # 朝向楼层
        dict_data['cx'] = "".join(root.xpath('//ul[@class="content__aside__list"]/li[3]//text()'))
        # =======房屋信息======
        # 面积
        dict_data['mj'] = root.xpath('//div[@class="content__article__info"]/ul[1]/li[2]/text()')[0]
        # 朝向
        dict_data['chaoxiang'] = root.xpath('//div[@class="content__article__info"]/ul[1]/li[3]/text()')[0]
        # 楼层
        dict_data['louceng'] = root.xpath('//div[@class="content__article__info"]/ul[1]/li[8]/text()')[0]
        # 电梯
        dict_data['dianti'] = root.xpath('//div[@class="content__article__info"]/ul[1]/li[9]/text()')[0]
        # 车位
        dict_data['chewei'] = root.xpath('//div[@class="content__article__info"]/ul[1]/li[11]/text()')[0]
        # 用水
        dict_data['shui'] = root.xpath('//div[@class="content__article__info"]/ul[1]/li[12]/text()')[0]
        # 用电
        dict_data['dian'] = root.xpath('//div[@class="content__article__info"]/ul[1]/li[14]/text()')[0]
        # 燃气
        dict_data['ranqi'] = root.xpath('//div[@class="content__article__info"]/ul[1]/li[15]/text()')[0]
        # 燃气
        dict_data['cainuan'] = root.xpath('//div[@class="content__article__info"]/ul[1]/li[17]/text()')[0]
        # 系统时间
        dict_data['sj'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        # self.insertItem('链家_二手房_深圳', dict_data)
        print(str(dict_data))

    def insertItem(self, tableName, data):
        my_set = db[tableName]
        my_set.insert_one(data)


if __name__ == '__main__':
    conn = MongoClient('127.0.0.1', 27017)
    db = conn["Test"]
    test_get = Test_Get()
    for p in range(1, 28):
        print(f"==============第【{p}】页==============")
        # url = f"https://bj.lianjia.com/zufang/pg{current}/"
        url = f"https://sz.lianjia.com/zufang/dapengxinqu/ab200301001000pg{p}/#contentList"
        area = "大鹏新区"
        test_get.get_contents(p, url, area)